{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = ['wikipedia-windows.json', 'news-windows.json', 'wikipedia-1word.json']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'w') as fopen_jsonl:\n",
    "    for f in files:\n",
    "        with open(f) as fopen:\n",
    "            data = json.load(fopen)\n",
    "        for l in data:\n",
    "            d = {\"translation\": {\"src\": l[0], \"tgt\": l[1], 'prefix': 'terjemah ke Jawi: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            d = {\"translation\": {\"src\": l[1], \"tgt\": l[0], 'prefix': 'terjemah ke Rumi: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            fopen_jsonl.flush()\n",
    "            \n",
    "    with open('jawi-rumi-news-full.train') as fopen:\n",
    "        for l in fopen:\n",
    "            l = json.loads(l)\n",
    "            d = {\"translation\": {\"src\": l[0], \"tgt\": l[1], 'prefix': 'terjemah ke Jawi: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            d = {\"translation\": {\"src\": l[1], \"tgt\": l[0], 'prefix': 'terjemah ke Rumi: '}}\n",
    "            fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            fopen_jsonl.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = glob('wiki-rumi-jawi-*.jsonl')\n",
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    for f in files:\n",
    "        with open(f) as fopen:\n",
    "            for l in fopen:\n",
    "                l = json.loads(l)\n",
    "                d = {\"translation\": {\"src\": l['malay'], \"tgt\": l['jawi'], 'prefix': 'terjemah ke Jawi: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                d = {\"translation\": {\"src\": l['jawi'], \"tgt\": l['malay'], 'prefix': 'terjemah ke Rumi: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                fopen_jsonl.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "!shuf train.json > shuffled-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"translation\": {\"src\": \"\\u0641\\u0648\\u062f \\u0646\\u064a\\u062a\\u0648\\u0648\\u0631\\u0642 \\u0627\\u0633\\u064a\\u0627 \\u0645\\u0631\\u0648\\u06a4\\u0627\\u0643\\u0646 \\u0633\\u0627\\u0644\\u0648\\u0631\\u0646 \\u062a\\u064a\\u0644\\u064a\\u06cf\\u064a\\u0634\\u064a\\u0646 \\u064a\\u06a0 \\u0645\\u06bd\\u064a\\u0627\\u0631\\u0643\\u0646 \\u062a\\u0646\\u062a\\u06a0 \\u0645\\u0627\\u0643\\u0646\\u0646 \\u062f\\u0627\\u0646 \\u0645\\u0627\\u0633\\u0642\\u0646 \\u062f\\u0631\\u064a \\u06a4\\u0631\\u06a4\\u0648\\u0633\\u062a\\u0627\\u0643\\u0627\\u0646 \\u0641\\u0648\\u062f \\u0646\\u064a\\u062a\\u0648\\u0648\\u0631\\u0642. \\u0641\\u0648\\u062f \\u0646\\u064a\\u062a\\u0648\\u0648\\u0631\\u0642 \\u0627\\u0633\\u064a\\u0627 \\u062f\\u0644\\u0646\\u0686\\u0631\\u0643\\u0646 \\u06a4\\u062f \\u0665 \\u062c\\u0648\\u0644\\u0627\\u064a \\u0662\\u0660\\u0661\\u0660 \\u062f \\u0633\\u062a\\u0627\\u0631\\u0647\\u0648\\u0628 \\u062a\\u06cf \\u062f \\u0646\\u06ac\\u0627\\u0631\\u0627 \\u0633\\u064a\\u06a0\\u0627\\u06a4\\u0648\\u0631\\u0627 \\u062f \\u0633\\u0627\\u0644\\u0648\\u0631\\u0646 \\u0664\\u0661\\u0667. \\u0633\\u0627\\u0644\\u0648\\u0631\\u0646 \\u0627\\u064a\\u0646 \\u062c\\u0648\\u06ac \\u0628\\u0648\\u0644\\u064a\\u0647 \\u062f\\u062f\\u0627\\u06a4\\u062a\\u064a \\u062f \\u0645\\u0644\\u064a\\u0633\\u064a\\u0627 \\u062f \\u0633\\u0627\\u0644\\u0648\\u0631\\u0646 \\u0627\\u0633\\u062a\\u0631\\u0648\\u060c \\u0627\\u064a\\u0627\\u064a\\u062a \\u06a4\\u0644\\u0627\\u062a\\u0641\\u0648\\u0631\\u0645 \\u062a\\u06cf \\u0628\\u0631\\u0628\\u0627\\u064a\\u0631 \\u064a\\u06a0 \\u062a\\u0631\\u0628\\u0633\\u0631 \\u062f \\u0631\\u0646\\u062a\\u0627\\u0648 \\u0627\\u064a\\u0646. \\u06a4\\u0631\\u0648\\u06ac\\u0631\\u0627\\u0645. \\u062c\\u062f\\u0648\\u0627\\u0644 \\u0633\\u064a\\u0627\\u0631\\u0646 \\u0641\\u0648\\u062f \\u0646\\u064a\\u062a\\u0648\\u0648\\u0631\\u0642 \\u0627\\u0633\\u064a\\u0627 \\u0641\\u0648\\u062f \\u0646\\u064a\\u062a\\u0648\\u0648\\u0631\\u0642 \\u0627\\u0633\\u064a\\u0627 \\u0647\\u062f. \\u0641\\u0648\\u062f \\u0646\\u064a\\u062a\\u0648\\u0648\\u0631\\u0642 \\u0627\\u0633\\u064a\\u0627 \\u0627\\u064a\\u0627\\u0644\\u0647 \\u0633\\u0627\\u0644\\u0648\\u0631\\u0646 \\u062f\\u0641\\u064a\\u0646\\u064a\\u0633\\u064a \\u062a\\u064a\\u06a0\\u06ac\\u064a \\ufd3f\\u0647\\u062f\\ufd3e \\u0661\\u0660\\u0668\\u0660\\u0627 \\u064a\\u06a0\", \"tgt\": \"Food Network Asia merupakan saluran televisyen yang menyiarkan tentang makanan dan masakan dari perpustakaan Food Network. Food Network Asia dilancarkan pada 5 Julai 2010 di StarHub TV di negara Singapura di saluran 417. Saluran ini juga boleh didapati di Malaysia di saluran Astro, iaitu platform TV berbayar yang terbesar di rantau ini. Program. Jadual siaran Food Network Asia Food Network Asia HD. Food Network Asia ialah saluran definisi tinggi (HD) 1080i yang\", \"prefix\": \"terjemah ke Rumi: \"}}\r\n",
      "{\"translation\": {\"src\": \"akauntan negara malaysia (janm), katanya.\", \"tgt\": \"\\u0627\\u0643\\u0627\\u0621\\u0648\\u0646\\u062a\\u0646 \\u0646\\u06ac\\u0627\\u0631\\u0627 \\u0645\\u0644\\u064a\\u0633\\u064a\\u0627 \\ufd3f\\u062c\\u0646\\u0645\\ufd3e\\u060c \\u0643\\u0627\\u062a\\u06bd.\", \"prefix\": \"terjemah ke Jawi: \"}}\r\n",
      "{\"translation\": {\"src\": \"Korfez merupakan sebuah daerah di Wilayah Kocaeli, Turki.\", \"tgt\": \"\\u0643\\u0648\\u0631\\u0641\\u064a\\u0632 \\u0645\\u0631\\u0648\\u06a4\\u0627\\u0643\\u0646 \\u0633\\u0628\\u0648\\u0627\\u0647 \\u062f\\u0627\\u064a\\u0631\\u0647 \\u062f \\u0648\\u0644\\u0627\\u064a\\u0647 \\u0643\\u0648\\u0686\\u0627\\u064a\\u0644\\u064a\\u060c \\u062a\\u0631\\u0643\\u064a\\u0627.\", \"prefix\": \"terjemah ke Jawi: \"}}\r\n",
      "{\"translation\": {\"src\": \"\\u06a4\\u0631\\u0627\\u0633\\u0627\\u0646 \\u062f\\u0627\\u0646 \\u0643\\u0644\\u0627\\u0643\\u0648\\u0627\\u0646 \\u062a\\u0646\\u06a4\\u0627 \\u0645\\u0646\\u062c\\u064a\\u062c\\u0633\\u0643\\u0646 \\u0643\\u0648\\u0627\\u0631\\u0633\\u0646 \\u0645\\u0631\\u064a\\u0643.\", \"tgt\": \"perasaan dan kelakuan tanpa menjejaskan kewarasan mereka.\", \"prefix\": \"terjemah ke Rumi: \"}}\r\n",
      "{\"translation\": {\"src\": \"Tampang sisinya lurus, lehernya besar, bahunya bercerun. Belakangnya panjang dengan suku-suku yang lereng dan abdomen yang berlipat. Kakinya panjang dan berisi ototnya. Panjang\", \"tgt\": \"\\u062a\\u0645\\u06a4\\u06a0 \\u0633\\u064a\\u0633\\u064a\\u06bd \\u0644\\u0648\\u0631\\u0648\\u0633\\u060c \\u0644\\u064a\\u0647\\u064a\\u0631\\u06bd \\u0628\\u0633\\u0631\\u060c \\u0628\\u0627\\u0647\\u0648\\u06bd \\u0628\\u0631\\u0686\\u0631\\u0648\\u0646. \\u0628\\u0644\\u0627\\u0643\\u06a0\\u06bd \\u06a4\\u0646\\u062c\\u06a0 \\u062f\\u06a0\\u0646 \\u0633\\u0648\\u0643\\u0648-\\u0633\\u0648\\u0643\\u0648 \\u064a\\u06a0 \\u0644\\u064a\\u0631\\u064a\\u06a0 \\u062f\\u0627\\u0646 \\u0627\\u0628\\u062f\\u0648\\u0645\\u0646 \\u064a\\u06a0 \\u0628\\u0631\\u0644\\u064a\\u06a4\\u062a. \\u0643\\u0627\\u0643\\u064a\\u06bd \\u06a4\\u0646\\u062c\\u06a0 \\u062f\\u0627\\u0646 \\u0628\\u0631\\u0627\\u064a\\u0633\\u064a \\u0627\\u0648\\u062a\\u0648\\u062a\\u06bd. \\u06a4\\u0646\\u062c\\u06a0\", \"prefix\": \"terjemah ke Jawi: \"}}\r\n"
     ]
    }
   ],
   "source": [
    "!head -n 5 shuffled-train.json"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
